# -*- coding: utf-8 -*-
import scrapy,sys,os,json,time
from pymongo import MongoClient
reload(sys).setdefaultencoding('UTF-8')
import logging
from pyquery import PyQuery as pq
import string
import random
import requests

class clSpider(scrapy.Spider):
        name = "cl"
        allowed_domains = ["tmkoo.com"]
        
        def __init__(self):
            self.apiKeyView = 'TEST_APISTORE'
            self.apiPasswordView = '0HTCv5hvPH'
            self.apiKey = 'app13012365483'
            self.apiPassword = '/RRkLJ5Cao'
            self.result = []
            self.count = 0
            self.maxNumList = []

        def getApiKeyPassword(self):
            mobile = ''.join([random.choice(string.digits) for _ in range(11)])
            url = 'http://api.tmkoo.com/app-reg.php?mobile=%s' %(mobile)
            res = requests.get(url)
            res.encode = 'utf-8'
            if res.status_code == 200 :
                info = json.loads(res.text)
                self.apiKey = info['apiKey']
                self.apiPassword = info['apiPassword'];
                print "Pwd generate successfully"
                return info
            else :
                print 'pwd generate fiald'

        def store(self,filename,data):
            with open(filename, 'w') as f:  
                json.dump(data,f)

        def closed(self,reason):
            #self.bk.insert(self.insertArray)
            self.store('r.json',self.result)
            self.maxNumList.sort()
            Num = self.maxNumList[-1:]
            print "vailed Num is %s,last No is %s" %(self.count,Num)
            

        def start_requests(self):#19000000
            for x in xrange(5000000,6000000):
                url = 'http://api.tmkoo.com/app-search.php?keyword=%s&apiKey=%s&apiPassword=%s&pageSize=15&condition=2' %(x,self.apiKey,self.apiPassword)
                request = scrapy.Request(url,meta={'regNo':str(x)},callback=self.parse)
                #print "start checking regNo.%s" %(x)
                yield request

        def parse(self,response):
            self.regNo = regNo = response.meta['regNo']
            if response.status == 200:
                info = json.loads(response.body)
                if len(info['msg'])>0:
                    print "parse_Search regNo.%s error msg is :%s" %(regNo,info['msg'])
                    self.getApiKeyPassword()
                    url = 'http://api.tmkoo.com/app-search.php?keyword=%s&apiKey=%s&apiPassword=%s&pageSize=15&condition=2' %(regNo,self.apiKey,self.apiPassword)
                    request = scrapy.Request(url,meta={'regNo':regNo},callback=self.parse)
                    yield request
                else:
                    if len(info['results'])>0:
                        intCls = info['results'][0]['intCls']
                        item={'regNo':regNo,'intCls':intCls}
                        self.result.append(item)
                        self.count = self.count+1
                        self.maxNumList.append(regNo)
                        print 'regNo:%s,intCls:%s, count is %d,request successfully' %(regNo,intCls,self.count)
                        if len(self.maxNumList)>1000:
                            self.maxNumList.sort()
                            Num = self.maxNumList[-1:]
                            print "checking is until Num.%s" %(Num)
                            del self.maxNumList[:]
                    else:
                        print "regNo:%s Is Not Existed" %(regNo)